diff --git a/swh/web/assets/src/bundles/browse/origin-search.js b/swh/web/assets/src/bundles/browse/origin-search.js
index aaa3036c..6c7df1b3 100644
--- a/swh/web/assets/src/bundles/browse/origin-search.js
+++ b/swh/web/assets/src/bundles/browse/origin-search.js
@@ -1,223 +1,223 @@
/**
* Copyright (C) 2018 The Software Heritage developers
* See the AUTHORS file at the top-level directory of this distribution
* License: GNU Affero General Public License version 3, or any later version
* See top-level LICENSE file for more information
*/
import {heapsPermute} from 'utils/heaps-permute';
import {handleFetchError} from 'utils/functions';
let originPatterns;
let perPage = 20;
let limit = perPage * 10;
let offset = 0;
let currentData = null;
let inSearch = false;
function fixTableRowsStyle() {
setTimeout(() => {
$('#origin-search-results tbody tr').removeAttr('style');
});
}
function clearOriginSearchResultsTable() {
$('#origin-search-results tbody tr').remove();
}
function populateOriginSearchResultsTable(data, offset) {
let localOffset = offset % limit;
if (data.length > 0) {
$('#swh-origin-search-results').show();
$('#swh-no-result').hide();
clearOriginSearchResultsTable();
let table = $('#origin-search-results tbody');
for (let i = localOffset; i < localOffset + perPage && i < data.length; ++i) {
let elem = data[i];
let tableRow = '
';
tableRow += '' + elem.type + ' | ';
let browseUrl = Urls.browse_origin(elem.url);
tableRow += '' + browseUrl + ' | ';
tableRow += ' | ';
tableRow += '
';
table.append(tableRow);
// get async latest visit snapshot and update visit status icon
let latestSnapshotUrl = Urls.browse_origin_latest_snapshot(elem.id);
fetch(latestSnapshotUrl)
.then(response => response.json())
.then(data => {
let originId = elem.id;
$('#visit-status-origin-' + originId).children().remove();
if (data) {
$('#visit-status-origin-' + originId).append('');
} else {
$('#visit-status-origin-' + originId).append('');
}
});
}
fixTableRowsStyle();
} else {
$('#swh-origin-search-results').hide();
$('#swh-no-result').text('No origins matching the search criteria were found.');
$('#swh-no-result').show();
}
if (data.length - localOffset < perPage ||
(data.length < limit && (localOffset + perPage) === data.length)) {
$('#origins-next-results-button').addClass('disabled');
} else {
$('#origins-next-results-button').removeClass('disabled');
}
if (offset > 0) {
$('#origins-prev-results-button').removeClass('disabled');
} else {
$('#origins-prev-results-button').addClass('disabled');
}
inSearch = false;
if (typeof Storage !== 'undefined') {
sessionStorage.setItem('last-swh-origin-search-offset', offset);
}
setTimeout(() => {
window.scrollTo(0, 0);
});
}
function escapeStringRegexp(str) {
let matchOperatorsRe = /[|\\{}()[\]^$+*?.]/g;
return str.replace(matchOperatorsRe, '\\\\\\$&');
}
function searchOrigins(patterns, limit, searchOffset, offset) {
originPatterns = patterns;
let patternsArray = patterns.trim().replace(/\s+/g, ' ').split(' ');
for (let i = 0; i < patternsArray.length; ++i) {
patternsArray[i] = escapeStringRegexp(patternsArray[i]);
}
let patternsPermut = [];
heapsPermute(patternsArray, p => patternsPermut.push(p.join('.*')));
let regex = patternsPermut.join('|');
let withVisit = $('#swh-search-origins-with-visit').prop('checked');
let searchUrl = Urls.browse_origin_search(regex) + `?limit=${limit}&offset=${searchOffset}®exp=true&with_visit=${withVisit}`;
clearOriginSearchResultsTable();
$('.swh-loading').addClass('show');
fetch(searchUrl)
.then(handleFetchError)
.then(response => response.json())
.then(data => {
currentData = data;
if (typeof Storage !== 'undefined') {
sessionStorage.setItem('last-swh-origin-url-patterns', patterns);
sessionStorage.setItem('last-swh-origin-search-results', JSON.stringify(data));
sessionStorage.setItem('last-swh-origin-search-offset', offset);
}
$('.swh-loading').removeClass('show');
populateOriginSearchResultsTable(data, offset);
})
.catch(response => {
$('.swh-loading').removeClass('show');
inSearch = false;
$('#swh-origin-search-results').hide();
response.text().then(errorDescription => {
$('#swh-no-result').text(`Error ${response.status}: ${response.statusText}.\n${errorDescription}`);
$('#swh-no-result').show();
});
});
}
export function initOriginSearch() {
$(document).ready(() => {
if (typeof Storage !== 'undefined') {
originPatterns = sessionStorage.getItem('last-swh-origin-url-patterns');
let data = sessionStorage.getItem('last-swh-origin-search-results');
offset = sessionStorage.getItem('last-swh-origin-search-offset');
if (data) {
$('#origins-url-patterns').val(originPatterns);
offset = parseInt(offset);
currentData = JSON.parse(data);
populateOriginSearchResultsTable(currentData, offset);
}
let withVisit = sessionStorage.getItem('last-swh-origin-with-visit');
if (withVisit !== null) {
$('#swh-search-origins-with-visit').prop('checked', JSON.parse(withVisit));
}
}
- $('#search_origins').submit(event => {
+ $('#swh-search-origins').submit(event => {
event.preventDefault();
$('#swh-no-result').hide();
let patterns = $('#origins-url-patterns').val();
offset = 0;
inSearch = true;
// first try to resolve a swh persistent identifier
let resolvePidUrl = Urls.resolve_swh_pid(patterns);
fetch(resolvePidUrl)
.then(handleFetchError)
.then(response => response.json())
.then(data => {
// pid has been successfully resolved,
// so redirect to browse page
window.location = data.browse_url;
})
.catch(response => {
// pid resolving failed
if (patterns.startsWith('swh:')) {
// display a useful error message if the input
// looks like a swh pid
response.json().then(data => {
$('#swh-origin-search-results').hide();
$('.swh-search-pagination').hide();
$('#swh-no-result').text(data.reason);
$('#swh-no-result').show();
});
} else {
// otherwise, proceed with origins search
$('#swh-origin-search-results').show();
$('.swh-search-pagination').show();
searchOrigins(patterns, limit, offset, offset);
}
});
});
$('#origins-next-results-button').click(event => {
if ($('#origins-next-results-button').hasClass('disabled') || inSearch) {
return;
}
inSearch = true;
offset += perPage;
if (!currentData || (offset >= limit && offset % limit === 0)) {
searchOrigins(originPatterns, limit, offset, offset);
} else {
populateOriginSearchResultsTable(currentData, offset);
}
event.preventDefault();
});
$('#origins-prev-results-button').click(event => {
if ($('#origins-prev-results-button').hasClass('disabled') || inSearch) {
return;
}
inSearch = true;
offset -= perPage;
if (!currentData || (offset > 0 && (offset + perPage) % limit === 0)) {
searchOrigins(originPatterns, limit, (offset + perPage) - limit, offset);
} else {
populateOriginSearchResultsTable(currentData, offset);
}
event.preventDefault();
});
$(document).on('shown.bs.tab', 'a[data-toggle="tab"]', e => {
if (e.currentTarget.text.trim() === 'Search') {
fixTableRowsStyle();
}
});
$(window).on('unload', () => {
if (typeof Storage !== 'undefined') {
sessionStorage.setItem('last-swh-origin-with-visit',
JSON.stringify($('#swh-search-origins-with-visit').prop('checked')));
}
});
});
}
diff --git a/swh/web/templates/browse/branches.html b/swh/web/templates/browse/branches.html
index 3b590228..ccc4782e 100644
--- a/swh/web/templates/browse/branches.html
+++ b/swh/web/templates/browse/branches.html
@@ -1,58 +1,58 @@
{% extends "./browse.html" %}
{% comment %}
Copyright (C) 2017-2018 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% block swh-browse-content %}
{% endblock %}
{% block swh-browse-after-content %}
{% if prev_branches_url or next_branches_url %}
{% endif %}
{% endblock %}
diff --git a/swh/web/templates/browse/origin-save.html b/swh/web/templates/browse/origin-save.html
index 37065320..574a86f2 100644
--- a/swh/web/templates/browse/origin-save.html
+++ b/swh/web/templates/browse/origin-save.html
@@ -1,109 +1,109 @@
{% extends "./layout.html" %}
{% comment %}
Copyright (C) 2018 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% block header %}
{{ block.super }}
-
+
{% endblock %}
{% block navbar-content %}
Save code now
{% endblock %}
{% block browse-content %}
You can contribute to extend the content of the Software Heritage archive by submitting an origin
save request. To do so, fill the required info in the form below:
- Origin type: the type of version control system the software origin is using.
Currently, the only supported type is git
, for origins using Git.
Soon, the following origin types will also be available to save into the archive:
- Origin url: the url of the remote repository for the software origin.
Once submitted, your save request can either be:
- accepted: a visit to the provided origin will then be scheduled by Software Heritage in order to
load its content into the archive as soon as possible
- rejected: the provided origin url is blacklisted and no visit will be scheduled
- put in pending state: a manual review will then be performed in order to determine if the
origin can be safely loaded or not into the archive
Once a save request has been accepted, you can follow its current status in the
submitted save requests list.
Request date |
Origin type |
Origin url |
Request status |
Save task status |
{% endblock %}
\ No newline at end of file
diff --git a/swh/web/templates/browse/search.html b/swh/web/templates/browse/search.html
index bfd1653e..fd4627c1 100644
--- a/swh/web/templates/browse/search.html
+++ b/swh/web/templates/browse/search.html
@@ -1,66 +1,66 @@
{% extends "./layout.html" %}
{% comment %}
Copyright (C) 2017-2018 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% load static %}
{% block navbar-content %}
Search archived software
{% endblock %}
{% block browse-content %}
-
Origin type |
Origin browse url |
Visit status |
Searching origins ...
No origins matching the search criteria were found.
{% endblock %}
diff --git a/swh/web/templates/includes/apidoc-header.html b/swh/web/templates/includes/apidoc-header.html
index c45644ef..60028c27 100644
--- a/swh/web/templates/includes/apidoc-header.html
+++ b/swh/web/templates/includes/apidoc-header.html
@@ -1,187 +1,187 @@
{% comment %}
Copyright (C) 2015-2018 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
Endpoint index
You can jump directly to the
endpoint index
, which lists all available API functionalities, or read on for more general information about the API.
Data model
The
Software Heritage project harvests publicly available source code by tracking software distribution channels such as
version control systems, tarball releases, and distribution packages.
All retrieved source code and related metadata are stored in the Software Heritage archive, that is conceptually a
Merkle DAG. All nodes in the graph are content-addressable, i.e., their node identifiers are computed by hashing their
content and, transitively, that of all nodes reachable from them; and no node or edge is ever removed from the graph: the
Software Heritage archive is an append-only data structure.
The following types of objects (i.e., graph nodes) can be found in the Software Heritage archive
(for more information see the
Software Heritage glossary):
-
Content: a specific version of a file stored in the archive, identified by its cryptographic hashes (currently:
SHA1, Git-like "salted" SHA1, SHA256). Note that content objects are nameless; their names are context-dependent
and stored as part of directory entries (see below).
Also known as: "blob"
-
Directory: a list of directory entries, where each entry can point to content objects ("file entries"),
revisions ("revision entries"), or transitively to other directories ("directory entries"). All entries
are associated to the local name of the entry (i.e., a relative path without any path separator) and permission metadata
(e.g., chmod value or equivalent).
-
Revision: a point in time snapshot of the content of a directory, together with associated development metadata
(e.g., author, timestamp, log message, etc).
Also known as: "commit".
-
Release: a revision that has been marked as noteworthy with a specific name (e.g., a version number), together
with associated development metadata (e.g., author, timestamp, etc).
Also known as: "tag"
-
Origin: an Internet-based location from which a coherent set of objects (contents, revisions, releases, etc.)
archived by Software Heritage has been obtained. Origins are currently identified by URLs.
-
Visit: the passage of Software Heritage on a given origin, to retrieve all source code and metadata available
there at the time. A visit object stores the state of all visible branches (if any) available at the origin at visit
time; each of them points to a revision object in the archive. Future visits of the same origin will create new visit
objects, without removing previous ones.
-
Person: an entity referenced by a revision as either the author or the committer of the corresponding change.
A person is associated to a full name and/or an email address.
Version
The current version of the API is
v1.
Warning: this version of the API is not to be considered stable yet. Non-backward compatible changes might happen
even without changing the API version number.
Schema
API access is over HTTPS.
All API endpoints are rooted at
https://archive.softwareheritage.org/api/1/.
Data is sent and received as JSON by default.
Example:
The response format can be overridden using the
Accept
request header. In particular,
Accept: text/html
(that web browsers send by default) requests HTML pretty-printing, whereas
Accept: application/yaml
requests YAML-encoded responses.
Example:
Parameters
Some API endpoints can be tweaked by passing optional parameters. For GET requests, optional parameters can be passed as
an HTTP query string.
The optional parameter
fields
is accepted by all endpoints that return dictionaries and can be used to restrict the list of fields returned
by the API, in case you are not interested in all of them. By default, all available fields are returned.
Example:
Errors
While API endpoints will return different kinds of errors depending on their own semantics, some error patterns are common
across all endpoints.
Sending malformed data, including syntactically incorrect object identifiers, will result in a
400 Bad Request
HTTP response. Example:
Requesting non existent resources will result in a
404 Not Found
HTTP response. Example:
Unavailability of the underlying storage backend will result in a
503 Service Unavailable
HTTP response.
Requests that might potentially return many items will be paginated.
Page size is set to a default (usually: 10 items), but might be overridden with the
per_page
query parameter up to a maximum (usually: 50 items). Example:
curl https://archive.softwareheritage.org/api/1/origin/1/visits/?per_page=2
To navigate through paginated results, a
Link
HTTP response header is available to link the current result page to the next one. Example:
curl -i https://archive.softwareheritage.org/api/1/origin/1/visits/?per_page=2 | grep ^Link:
Link: </api/1/origin/1/visits/?last_visit=2&per_page=2>; rel="next",
Rate limiting
Due to limited resource availability on the back end side, API usage is currently rate limited. Furthermore, as API usage
is currently entirely anonymous (i.e., without any authentication), API "users" are currently identified by their
origin IP address.
Three HTTP response fields will inform you about the current state of limits that apply to your current rate limiting bucket:
-
X-RateLimit-Limit
: maximum number of permitted requests per hour
-
X-RateLimit-Remaining
: number of permitted requests remaining before the next reset
-
X-RateLimit-Reset
: the time (expressed in
Unix time seconds) at which the current rate limiting will expire, resetting to a fresh
X-RateLimit-Limit
Example:
curl -i https://archive.softwareheritage.org/api/1/stat/counters/ | grep ^X-RateLimit
X-RateLimit-Limit: 60
X-RateLimit-Remaining: 54
X-RateLimit-Reset: 1485794532
\ No newline at end of file